In [ ]:
from planet4 import markings, dbscan, io, fnotching
In [ ]:
from nbtools.logging import setup_live_logging
import logging
In [ ]:
setup_live_logging('planet4', logging.DEBUG)
In [ ]:
from planet4 import region_data, io
In [ ]:
db = io.DBManager()
In [ ]:
obsids = db.image_names
In [ ]:
db.dbname
In [ ]:
len(obsids)
In [ ]:
from planet4.catalog_production import ReleaseManager
from planet4 import fnotching
In [ ]:
rm = ReleaseManager("v1.1")
In [ ]:
rm.launch_catalog_production()
In [ ]:
rm.merge_all()
In [ ]:
pd.read_csv(rm.blotch_merged).shape
In [ ]:
pd.read_csv(rm.fan_merged).shape
In [ ]:
fans = pd.read_csv(self.fan_file)
blotches = pd.read_csv(self.blotch_file)
meta = pd.read_csv(self.metadata_path, dtype='str')
tile_coords = pd.read_csv(self.tile_coords_path, dtype='str')
In [ ]:
meta.shape
In [ ]:
tile_coords.shape
In [ ]:
cols_to_merge = ['OBSERVATION_ID',
'SOLAR_LONGITUDE', 'north_azimuth', 'map_scale']
fans = fans.merge(meta[cols_to_merge],
left_on='obsid', right_on='OBSERVATION_ID')
blotches = blotches.merge(
meta[cols_to_merge], left_on='obsid', right_on='OBSERVATION_ID')
In [ ]:
fans.shape
In [ ]:
blotches.shape
In [ ]:
fans.drop(self.DROP_FOR_FANS, axis=1, inplace=True)
blotches.drop(self.DROP_FOR_BLOTCHES, axis=1, inplace=True)
In [ ]:
blotches.shape
In [ ]:
fans, blotches = self.merge_fnotch_results(fans, blotches)
In [ ]:
fans.shape
In [ ]:
blotches.shape
In [ ]:
from planet4 import catalog_production as cp
In [ ]:
rm.EDRINDEX_meta_path
In [ ]:
rm.merge_all()
In [ ]:
rm.calc_metadata()
In [ ]:
meta = pd.read_csv(rm.metadata_path).set_index('OBSERVATION_ID')
In [ ]:
rm.EDRINDEX_meta_path
In [ ]:
meta.loc['ESP_012251_0935']
In [ ]:
edrindex = pd.read_hdf("/Volumes/Data/hirise/EDRCUMINDEX.hdf")
p4_edr = edrindex[edrindex.OBSERVATION_ID.isin(rm.obsids)].query(
'CCD_NAME=="RED4"').drop_duplicates(subset='OBSERVATION_ID')
In [ ]:
p4_edr.set_index('OBSERVATION_ID', inplace=True)
In [ ]:
p4_edr.loc['ESP_012251_0935']
In [ ]:
In [ ]:
rm.calc_metadata()
In [ ]:
rm.
In [ ]:
df.set_index('OBSERVATION_ID').to_csv(rm.EDRINDEX_meta_path, index=True)
In [ ]:
cols = ['OBSERVATION_ID', 'IMAGE_CENTER_LATITUDE', 'IMAGE_CENTER_LONGITUDE', 'SOLAR_LONGITUDE', 'START_TIME',
'north_azimuth', '# of tiles']
In [ ]:
df[cols].head()
In [ ]:
fname = "P4_catalog_v1.0_EDRINDEX_metadata.csv"
df.to_csv(rm.savefolder / fname, index=False)
In [ ]:
rm.savefolder
In [ ]:
import itertools
import string
# Adapt this prefix as a unique identifier of the dataset
def fan_id_generator():
for newid in itertools.product(string.digits+'abcdef', repeat=6):
yield 'F' + ''.join(newid)
def blotch_id_generator():
for newid in itertools.product(string.digits+'abcdef', repeat=6):
yield 'B' + ''.join(newid)
In [ ]:
def add_marking_ids(path, fan_id, blotch_id):
"""Add marking_ids for catalog to cluster results.
Parameters
----------
path : str, pathlib.Path
Path to L1A image_id clustering result directory
fan_id, blotch_id : generator
Generator for marking_id
"""
image_id = path.parent.name
for kind, id_ in zip(['fans', 'blotches'], [fan_id, blotch_id]):
fname = str(path / f"{image_id}_L1A_{kind}.csv")
try:
df = pd.read_csv(fname)
except FileNotFoundError:
continue
else:
marking_ids = []
for i in range(df.shape[0]):
marking_ids.append(next(id_))
df['marking_id'] = marking_ids
df.to_csv(fname, index=False)
In [ ]:
from tqdm import tqdm
In [ ]:
obsids = rm.obsids
In [ ]:
rm.launch_catalog_production()
In [ ]:
rm.calc_metadata()
In [ ]:
rm.calc_marking_coordinates()
In [ ]:
rm.merge_all()
In [ ]:
fan_id = fan_id_generator()
blotch_id = blotch_id_generator()
In [ ]:
from planet4.fnotching import fnotch_image_ids, apply_cut
In [ ]:
for obsid in tqdm(obsids):
pm = io.PathManager(obsid=obsid, datapath=rm.savefolder)
paths = pm.get_obsid_paths('L1A')
for path in paths:
add_marking_ids(path, fan_id, blotch_id)
In [ ]:
rm.catalog
In [ ]:
for obsid in tqdm(obsids):
fnotch_image_ids(obsid, savedir=rm.catalog)
apply_cut(obsid, savedir=rm.catalog)
In [ ]:
from planet4.catalog_production import create_roi_file
In [ ]:
create_roi_file(obsids, rm.catalog, rm.catalog)
In [ ]:
rm.get_metadata()
In [ ]:
rm.merge_all()
In [ ]:
fans = rm.read_fan_file()
In [ ]:
fans.columns
In [ ]:
blotches = rm.read_blotch_file()
In [ ]:
blotches.columns
In [ ]:
pd.read_csv(rm.metadata_path).columns
In [ ]:
fans.vote_ratio.isna().value_counts()
In [ ]:
fans.head()
In [ ]:
rm.launch_catalog_production()
In [ ]:
from planet4.fnotching import get_clusters_in_path
def find_bad_obsid(obsid, savedir):
pm = io.PathManager(obsid=obsid, datapath=savedir)
paths = pm.get_obsid_paths('L1A')
if len(paths) == 0:
logger.warning("No paths to fnotch found for %s", obsid)
fans = []
blotches = []
for path in paths:
f, b = get_clusters_in_path(path)
fans.append(f)
blotches.append(b)
fans = pd.concat(fans, ignore_index=True)
In [ ]:
folder
In [ ]:
for obsid in obsids:
print(obsid)
find_bad_obsid(obsid, folder)
In [ ]:
import holoviews as hv
hv.notebook_extension()
In [ ]:
from planet4.io import get_subframe
In [ ]:
obsid = 'ESP_022273_0950'
In [ ]:
data = db.get_image_name_markings(obsid)[['image_id', 'image_url']]
In [ ]:
data.set_index('image_id', drop=True, inplace=True)
In [ ]:
data.drop_duplicates(inplace=True)
In [ ]:
def show_subframe(image_id):
img = get_subframe(data.loc[data.index[0]].values[0])
return hv.Raster(img)
In [ ]:
%output size=150
In [ ]:
%%opts Raster [xaxis=None, yaxis=None]
show_subframe(data.index[0])+show_subframe(data.index[1])
In [ ]:
dmap = hv.DynamicMap(show_subframe, kdims=['image_id'])
dmap
In [ ]:
dmap.redim.values(image_id=data.index.values)
In [ ]:
xvals = np.linspace(-4,0,202)
yvals = np.linspace(4,0,202)
xs,ys = np.meshgrid(xvals, yvals)
def waves_image(alpha, beta):
return hv.Image(np.sin(((ys/alpha)**alpha+beta)*xs))
waves_image(0,0) + waves_image(0,4)
In [ ]:
dmap = hv.DynamicMap(waves_image, kdims=['alpha', 'beta'])
dmap
In [ ]:
dmap[0,1] + dmap.select(alpha=1, beta=2)
In [ ]:
dmap.redim.range(alpha=(0,5.0), beta=(1,5.0))
In [ ]:
hv.__version__
In [ ]:
for imgid in image_ids:
do_cluster_ids(imgid, "without_large_fan_fnotch_on_p4_coords" )
In [ ]:
obsid = 'ESP_020930_0980'
savedir = 'fnotching_per_obsid'
In [ ]:
dbscanner = dbscan.DBScanner(savedir=savedir)
In [ ]:
dbscanner.cluster_image_name(obsid)
In [ ]:
pm.final_blotchfile
In [ ]:
fnotching.fnotch_obsid(obsid, savedir=savedir)
In [ ]:
fnotching.apply_cut_obsid(obsid, savedir=savedir)
In [ ]:
In [ ]:
In [ ]:
data = region_data.Inca.season2 + region_data.Inca.season3
In [ ]:
do_clustering(ithaca[0])
In [ ]:
ithaca[0]
In [ ]:
from nbtools import execute_in_parallel
In [ ]:
result = execute_in_parallel(do_clustering, data)
In [ ]:
from planet4.fnotching import get_clusters_in_path
In [ ]:
pm = io.PathManager(obsid=roi.seasons[0], datapath=folder)
In [ ]:
pm.obsid_final_blotches_path
In [ ]:
from planet4.catalog_production import create_roi_file
In [ ]:
savedir
In [ ]:
create_roi_file(obsids, 'catalog', savedir)
In [ ]:
pm = io.PathManager(obsid=obsids[0], datapath=savedir)
In [ ]:
folders = pm.get_obsid_paths('L1C')
In [ ]:
from planet4.catalog_production import read_csvfiles_into_lists_of_frames, concat_bucket_lists
In [ ]:
bucket.items()
In [ ]:
In [ ]:
roi.available_seasons
In [ ]:
folder
In [ ]:
def create_roi_file_with_seasons(roi, datapath, copy_to_analysis=True):
Fans = []
Blotches = []
for season in roi.available_seasons:
print(season)
# keep lower case marking objects to add season column later
fans = []
blotches = []
for obsid in getattr(roi, season):
print(obsid)
pm = io.PathManager(obsid=obsid, datapath=datapath)
fans.append(pm.final_fandf)
blotches.append(pm.final_blotchdf)
fans = pd.concat(fans, ignore_index=True)
fans['season'] = season
Fans.append(fans)
blotches = pd.concat(blotches, ignore_index=True)
blotches['season'] = season
Blotches.append(blotches)
Fans = pd.concat(Fans, ignore_index=True)
Blotches = pd.concat(Blotches, ignore_index=True)
savedir = pm.path_so_far.parent
for marks in ['Fans', 'Blotches']:
savename = f"{roi.name}_{pm.L1C_folder}_{marks}.csv"
savepath = savedir / savename
obj = eval(marks) # get handle to the fan or blotch object
obj = obj.dropna(how='all', axis=1)
obj.to_csv(savepath, index=False)
if copy_to_analysis is True:
savefolder = io.analysis_folder() / folder / f"{roi.name.lower()}"
savefolder.mkdir(exist_ok=True)
savepath = savefolder / savename
obj.to_csv(savepath, index=False)
In [ ]:
create_roi_file_with_seasons(roi, folder)
In [ ]:
def create_obsid_result_files(obsid, datapath=None):
# logger.debug("Working on %s", obsid)
pm = io.PathManager(obsid=obsid, datapath=datapath)
# the pm.get_obsid_paths method searches for existing image_id folders inside each
# obsid folder
folders = list(pm.get_obsid_paths(pm.L1C_folder))
blotches = []
fans = []
for folder in folders:
pm.id = fnotching.get_id_from_path(folder)
if pm.final_blotchfile.exists() and pm.final_blotchfile.lstat().st_size > 1:
blotches.append(pm.final_blotchdf)
if pm.final_fanfile.exists() and pm.final_fanfile.lstat().st_size > 1:
fans.append(pm.final_fandf)
if len(blotches) > 0:
blotches = pd.concat(blotches, ignore_index=True)
blotches.dropna(how='all', axis=1, inplace=True)
blotches.to_csv(pm.obsid_final_blotches_path, index=False)
print("Created", pm.obsid_final_blotches_path)
if len(fans) > 0:
fans = pd.concat(fans, ignore_index=True)
fans.dropna(how='all', axis=1, inplace=True)
fans.to_csv(pm.obsid_final_fans_path, index=False)
print("Created", pm.obsid_final_fans_path)
In [ ]:
for obsid in data:
create_obsid_result_files(obsid, datapath='for_chase')
In [ ]:
region = getattr(region_data, name)
for season_no in [2, 3]:
season = str(season_no)
bucket = []
for obsid in getattr(region, f"season{season}"):
pm = io.PathManager(obsid=obsid)
df = pd.read_csv(pm.obsid_final_blotches_path)
df['obsid'] = obsid
bucket.append(df)
df = pd.concat(bucket, ignore_index=True)
df.to_csv(io.analysis_folder() / f"p4_catalog/{name}_season{season}_blotches.csv",
index=False)
In [ ]:
store_folder = io.analysis_folder() / 'for_chase'
store_folder.mkdir(exist_ok=True)
In [ ]:
name = 'inca'
In [ ]:
p = Path('/Users/klay6683/Dropbox/data/planet4/for_chase/' + name)
for kind in ['blotches', 'fans']:
bucket = []
filepaths = p.glob(f'*_{kind}.csv')
for filepath in filepaths:
obsid = filepath.name[:15]
df = pd.read_csv(filepath)
df['obsid'] = obsid
bucket.append(df)
df = pd.concat(bucket, ignore_index=True)
savedir = store_folder / f'{name}'
savedir.mkdir(exist_ok=True)
savepath = savedir / f"{name}_{kind}.csv"
print(savepath)
df.to_csv(savepath, index=False)
In [ ]: